{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import h5py\n",
    "import glob\n",
    "import numbers\n",
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/media/luiza/HDLuiza/Dataset/Gaze/MPIIFaceGaze_normalizad/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source: https://github.com/fab-jul/hdf5_dataloader\n",
    "\n",
    "default_opener = lambda p_: h5py.File(p_, 'r')\n",
    "\n",
    "class HDF5Dataset(Dataset):\n",
    "    def __init__(self, file_ps,\n",
    "                 custom_getitem,\n",
    "                 files_and_shards,\n",
    "                 transform=None,\n",
    "                 transform_label=None,\n",
    "                 shuffle_shards=True,\n",
    "                 opener=default_opener,\n",
    "                 seed=42):\n",
    "        \"\"\"\n",
    "        Parameters:\n",
    "            - file_ps: list of file paths to .hdf5 files. Last (alphabetically) \n",
    "            file is expected to contain lessimages.\n",
    "            - custom_getitem: custom function defined by user to get data from\n",
    "            the hdf5 file. It's inputs are be opened hdf5 file and sampled index.\n",
    "            - files_and_shards: dictionary containing entries such as \n",
    "                    {file1: num_shards1, ..., fileN: num_shardsN}\n",
    "            - transform: transformation to apply to read HDF5 sample.\n",
    "            - transform_label: tranformation to apply to read HDF5 label.\n",
    "            - shuffle_shards: if true, shards are shuffled with seed\n",
    "        \"\"\"\n",
    "\n",
    "        if len(file_ps) == 0 or not all(os.path.isfile(p) for p in file_ps):\n",
    "            raise ValueError('Expected list of paths to HDF5 files, got {}'.format(file_ps))\n",
    "        self.opener = opener\n",
    "        self.ps, self.num_per_shard = HDF5Dataset.filter_smaller_shards(file_ps)\n",
    "        if shuffle_shards:\n",
    "            r = random.Random(seed)\n",
    "            r.shuffle(self.ps)\n",
    "        self.transform = transform\n",
    "        self.transform_label = transform_label\n",
    "        \n",
    "        ## Custom function defined by user to handle\n",
    "        # return of data from hdf5 file.\n",
    "        self.custom_getitem = custom_getitem \n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.ps) * self.num_per_shard\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        shard_idx = index // self.num_per_shard\n",
    "        idx_in_shard = index % self.num_per_shard\n",
    "        shard_p = self.ps[shard_idx]\n",
    "        with self.opener(shard_p) as f:\n",
    "            item = self.custom_getitem(f, idx_in_shard)\n",
    "        if self.transform is not None:\n",
    "            item['sample'] = self.transform(img)\n",
    "        if self.transform_label is not None:\n",
    "            item['label'] = self.transform(label)\n",
    "        return item\n",
    "\n",
    "    @staticmethod\n",
    "    def filter_smaller_shards(file_ps, opener=default_opener):\n",
    "        \"\"\"\n",
    "        Filter away the (alphabetically) last shard, which is assumed to be smaller. This function also double checks\n",
    "        that all other shards have the same number of entries.\n",
    "        Parameters:\n",
    "            - file_ps: list of .hdf5 files, does not have to be sorted.\n",
    "            - opener: function to open shards\n",
    "        Return:\n",
    "            - tuple (ps, num_per_shard), where\n",
    "                -> ps = filtered file paths,\n",
    "                -> num_per_shard = number of entries in all of the shards in `ps`\n",
    "        \"\"\"\n",
    "        assert file_ps, 'No files given'\n",
    "        file_ps = sorted(file_ps)  # we assume that smallest shard is at the end\n",
    "        num_per_shard_prev = None\n",
    "        ps = []\n",
    "        for i, p in enumerate(file_ps):\n",
    "            num_per_shard = get_num_in_shard(p, files_and_shards, opener)\n",
    "            if num_per_shard_prev is None:  # first file\n",
    "                num_per_shard_prev = num_per_shard\n",
    "                ps.append(p)\n",
    "                continue\n",
    "            if num_per_shard_prev < num_per_shard:\n",
    "                raise ValueError('Expected all shards to have the same number of elements,'\n",
    "                                 'except last one. Previous had {} elements, current ({}) has {}!'.format(\n",
    "                                    num_per_shard_prev, p, num_per_shard))\n",
    "            if num_per_shard_prev > num_per_shard:  # assuming this is the last\n",
    "                is_last = i == len(file_ps) - 1\n",
    "                if not is_last:\n",
    "                    raise ValueError(\n",
    "                            'Found shard with too few elements, and it is not the last one! {}\\n'\n",
    "                            'Last: {}\\n'.format(p, file_ps[-1]))\n",
    "                print('Filtering shard {}, dropping {} elements...'.format(p, num_per_shard))\n",
    "                break  # is last anyways\n",
    "            else:  # same numer as before, all good\n",
    "                ps.append(p)\n",
    "        return ps, num_per_shard_prev\n",
    "    \n",
    "def get_num_in_shard(shard_p, files_and_shards, opener=default_opener):\n",
    "    hdf5_root = os.path.dirname(shard_p)\n",
    "    if files_and_shards[os.path.basename(shard_p)]:\n",
    "        num_per_shard = files_and_shards[os.path.basename(shard_p)]\n",
    "    else:\n",
    "        print('\\rOpening {}...'.format(shard_p), end='')\n",
    "        with opener(shard_p) as f:\n",
    "            num_per_shard = len(f.keys())\n",
    "    return num_per_shard\n",
    "\n",
    "def get_loaders(files_and_shards, files_path, test_filenames, custom_getitem, \n",
    "                extension='.h5', transform=None, transform_label=None,\n",
    "                batch_size=16, num_workers=8, use_gpu=True):\n",
    "    \"\"\"\n",
    "        Parameters:\n",
    "            - files_and_shards: dictionary containing entries such as \n",
    "                    {file1: num_shards1, ..., fileN: num_shardsN}.\n",
    "            This means it's the filename and the amount of data in it. It's \n",
    "            expected the last file to have fewer entries, and the rest have \n",
    "            the same amount of data.\n",
    "            - files_path: path to folder where files are (in this case, all files\n",
    "            are in the same folder, not separated in train/test folders).\n",
    "            - test_filenames: which files are to be separated for validation/test.\n",
    "            - custom_getitem: custom function defined by user to get data from\n",
    "            the hdf5 file. It's inputs are be opened hdf5 file and sampled index.\n",
    "            - extension: file extension for hdf5 file ('.h5', '.hdf5', ...)\n",
    "            - transform and transform_label: custom data transforms.\n",
    "            - batch_size: PyTorch dataloader batch_size.\n",
    "            - num_workers: PyTorch's dataloader num_workers param.\n",
    "            - use_gpu: PyTorch's dataloader pin_memory param.\n",
    "    \"\"\"\n",
    "\n",
    "    train_files = glob.glob(files_path + '*' + extension)\n",
    "    \n",
    "    test_files = [file for file in train_files if any(f in file for f in test_filenames)]\n",
    "    \n",
    "    train_files = [file for file in train_files if file not in test_files]\n",
    "    \n",
    "    train_dataset = HDF5Dataset(train_files, custom_getitem=getitem_func, \n",
    "                 files_and_shards=files_and_shards, transform=transform,\n",
    "                 transform_label=transform_label)\n",
    "    \n",
    "    test_dataset = HDF5Dataset(test_files, custom_getitem=getitem_func, \n",
    "                 files_and_shards=files_and_shards, transform=transform,\n",
    "                 transform_label=transform_label)\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, \n",
    "                              batch_size=batch_size, \n",
    "                              shuffle=True, \n",
    "                              num_workers=num_workers,\n",
    "                              pin_memory=use_gpu, \n",
    "                              drop_last=True)\n",
    "    \n",
    "    test_loader = DataLoader(test_dataset, \n",
    "                             batch_size=batch_size, \n",
    "                             shuffle=False, \n",
    "                             num_workers=num_workers,\n",
    "                             pin_memory=use_gpu, \n",
    "                             drop_last=False)\n",
    "    \n",
    "    del train_dataset, test_dataset # let's clear all memory we can lol\n",
    "    \n",
    "    return train_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getitem_func(file, index):\n",
    "    \"\"\"\n",
    "        Return image and labels from MPIIFaceGaze with correct shapes.\n",
    "        PyTorch needs the image to be [channel x width x height].\n",
    "        The images came as BGR, so there was a need to invert it \n",
    "        to RGB also.\n",
    "    \"\"\"\n",
    "    img = np.transpose(file['data'][index][[2, 1, 0], :, :], (1, 2, 0))\n",
    "    label = file['label'][index]\n",
    "    return {'sample': img, 'label': label}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create transform\n",
    "# Note: cannot use default PyTorch ops, because they expect PIL Images\n",
    "\n",
    "# all files and its respective shards\n",
    "files_and_shards = {'p{:02}_{}.h5'.format(i, j):1500 for j in range(2) for i in range(15)}\n",
    "\n",
    "# get all files paths from main folder\n",
    "all_file_ps = glob.glob(path + '*.h5')\n",
    "\n",
    "# create dataset\n",
    "ds = HDF5Dataset(all_file_ps, custom_getitem=getitem_func, \n",
    "                 files_and_shards=files_and_shards, transform=None)\n",
    "    \n",
    "# using the standard PyTorch DataLoader\n",
    "dl = DataLoader(ds, batch_size=1, shuffle=True, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_image(image):\n",
    "    plt.imshow(image, interpolation='nearest')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'builtin_function_or_method' object is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-331-aa02bba77a2a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sample'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sample'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;31m# PyTorch needs the image to be channel x width x height\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;31m# So, to display the image, we have to transpose the channels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    613\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_workers\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# same-process loading\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    614\u001b[0m             \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample_iter\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 615\u001b[0;31m             \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    616\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    617\u001b[0m                 \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpin_memory_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    613\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_workers\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# same-process loading\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    614\u001b[0m             \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample_iter\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 615\u001b[0;31m             \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    616\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    617\u001b[0m                 \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpin_memory_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-323-01ef4857a5a7>\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcustom_getitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx_in_shard\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m             \u001b[0mitem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sample'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform_label\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0mitem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'label'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransforms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m    236\u001b[0m             \u001b[0mPIL\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mCropped\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m         \"\"\"\n\u001b[0;32m--> 238\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcenter_crop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torchvision/transforms/functional.py\u001b[0m in \u001b[0;36mcenter_crop\u001b[0;34m(img, output_size)\u001b[0m\n\u001b[1;32m    372\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumbers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m         \u001b[0moutput_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 374\u001b[0;31m     \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    375\u001b[0m     \u001b[0mth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    376\u001b[0m     \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m2.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'builtin_function_or_method' object is not iterable"
     ]
    }
   ],
   "source": [
    "for batch in dl:\n",
    "    if batch['sample'] is not None:\n",
    "        print(batch['sample'][0, :, :, :].shape)\n",
    "        # PyTorch needs the image to be channel x width x height\n",
    "        # So, to display the image, we have to transpose the channels\n",
    "        show_image(batch['sample'][0, :, :, :].squeeze().numpy().astype(np.int32))\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader, test_loader = get_loaders(files_and_shards, path, ['p05_0', 'p07_0', 'p12_1'], getitem_func, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check size of last file, which is smaller than the others\n",
    "with default_opener(path+'p15_0.h5') as f:\n",
    "    print(f['label'].shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
